package engine

import (
	"clawer/service/fetcher"
	"log"
)

type SimpleEngine struct{}

var (
	fetch fetcher.Service
)

func init() {
	fetch = fetcher.NewService()
}

func (e SimpleEngine) Run(seeds ...Request) {
	var requests []Request
	for _, r := range seeds {
		requests = append(requests, r)
	}
	for len(requests) > 0 {
		r := requests[0]
		requests = requests[1:]
		parseResult, err := e.worker(r)
		if err != nil {
			continue
		}
		requests = append(requests,
			parseResult.Requests...)
		for _, item := range parseResult.Items {
			log.Printf("Got item :%v", item)
		}
	}
}

func (e SimpleEngine) worker(r Request) (ParseResult, error) {
	body, err := fetch.Fetch(r.Url)
	if err != nil {
		log.Printf("Fetcher: error fetching url :%s :%v", r.Url, err)
		return ParseResult{}, err
	}
	return r.ParserFunc(body), nil
}
